{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "12843c31",
   "metadata": {},
   "source": [
    "## 测试AutoAugmentation的作用"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c2870f64",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Use device:  cuda\n"
     ]
    }
   ],
   "source": [
    "# 自动重新加载外部module，使得修改代码之后无需重新import\n",
    "# see http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "from hdd.device.utils import get_device\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms\n",
    "\n",
    "# 设置训练数据的路径\n",
    "DATA_ROOT = \"~/workspace/hands-dirty-on-dl/dataset\"\n",
    "# 设置TensorBoard的路径\n",
    "TENSORBOARD_ROOT = \"~/workspace/hands-dirty-on-dl/dataset\"\n",
    "# 设置预训练模型参数路径\n",
    "TORCH_HUB_PATH = \"~/workspace/hands-dirty-on-dl/pretrained_models\"\n",
    "torch.hub.set_dir(TORCH_HUB_PATH)\n",
    "# 挑选最合适的训练设备\n",
    "DEVICE = get_device([\"cuda\", \"cpu\"])\n",
    "print(\"Use device: \", DEVICE)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fca09cb9",
   "metadata": {},
   "source": [
    "### Baseline Data Augmentation\n",
    "\n",
    "```python\n",
    "transforms.RandomCrop(size=32, padding=4),\n",
    "transforms.RandomHorizontalFlip(),\n",
    "transforms.ToTensor(),\n",
    "transforms.Normalize(mean=TRAIN_MEAN, std=TRAIN_STD),\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8f0430be",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "# 我们提前计算好了训练数据集上的均值和方差\n",
    "TRAIN_MEAN = [0.49139968, 0.48215827, 0.44653124]\n",
    "TRAIN_STD = [0.24703233, 0.24348505, 0.26158768]\n",
    "\n",
    "train_dataset_transforms = transforms.Compose(\n",
    "    [\n",
    "        transforms.RandomCrop(size=32, padding=4),\n",
    "        transforms.RandomHorizontalFlip(),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize(mean=TRAIN_MEAN, std=TRAIN_STD),\n",
    "    ]\n",
    ")\n",
    "# 加载数据集\n",
    "train_dataset = datasets.CIFAR10(\n",
    "    root=DATA_ROOT,\n",
    "    train=True,\n",
    "    transform=train_dataset_transforms,\n",
    "    download=True,\n",
    ")\n",
    "val_dataset = datasets.CIFAR10(\n",
    "    root=DATA_ROOT,\n",
    "    train=False,\n",
    "    transform=transforms.Compose(\n",
    "        [transforms.ToTensor(), transforms.Normalize(TRAIN_MEAN, TRAIN_STD)]\n",
    "    ),\n",
    "    download=True,\n",
    ")\n",
    "BATCH_SIZE = 128\n",
    "train_dataloader = torch.utils.data.DataLoader(\n",
    "    train_dataset,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    shuffle=True,\n",
    "    num_workers=4,\n",
    ")\n",
    "val_dataloader = torch.utils.data.DataLoader(\n",
    "    val_dataset,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    shuffle=False,\n",
    "    num_workers=4,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6e57da60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1/160 Train Loss: 1.5633 Accuracy: 0.4172 Time: 10.65951  | Val Loss: 1.3756 Accuracy: 0.5193\n",
      "Epoch: 2/160 Train Loss: 1.0187 Accuracy: 0.6349 Time: 10.68677  | Val Loss: 0.9361 Accuracy: 0.6767\n",
      "Epoch: 3/160 Train Loss: 0.7805 Accuracy: 0.7261 Time: 10.45274  | Val Loss: 0.7739 Accuracy: 0.7284\n",
      "Epoch: 4/160 Train Loss: 0.6293 Accuracy: 0.7789 Time: 10.32677  | Val Loss: 0.6849 Accuracy: 0.7699\n",
      "Epoch: 5/160 Train Loss: 0.5348 Accuracy: 0.8125 Time: 10.46105  | Val Loss: 0.6484 Accuracy: 0.7899\n",
      "Epoch: 6/160 Train Loss: 0.4730 Accuracy: 0.8359 Time: 10.63133  | Val Loss: 0.4966 Accuracy: 0.8314\n",
      "Epoch: 7/160 Train Loss: 0.4143 Accuracy: 0.8584 Time: 10.47751  | Val Loss: 0.4930 Accuracy: 0.8345\n",
      "Epoch: 8/160 Train Loss: 0.3785 Accuracy: 0.8675 Time: 10.39104  | Val Loss: 0.4361 Accuracy: 0.8524\n",
      "Epoch: 9/160 Train Loss: 0.3345 Accuracy: 0.8840 Time: 10.60739  | Val Loss: 0.4769 Accuracy: 0.8496\n",
      "Epoch: 10/160 Train Loss: 0.3101 Accuracy: 0.8922 Time: 10.60040  | Val Loss: 0.3983 Accuracy: 0.8661\n",
      "Epoch: 11/160 Train Loss: 0.2832 Accuracy: 0.9017 Time: 10.36479  | Val Loss: 0.3787 Accuracy: 0.8761\n",
      "Epoch: 12/160 Train Loss: 0.2557 Accuracy: 0.9095 Time: 10.53740  | Val Loss: 0.4775 Accuracy: 0.8508\n",
      "Epoch: 13/160 Train Loss: 0.2390 Accuracy: 0.9167 Time: 10.49811  | Val Loss: 0.3643 Accuracy: 0.8821\n",
      "Epoch: 14/160 Train Loss: 0.2164 Accuracy: 0.9246 Time: 10.58889  | Val Loss: 0.3516 Accuracy: 0.8853\n",
      "Epoch: 15/160 Train Loss: 0.1994 Accuracy: 0.9297 Time: 10.56551  | Val Loss: 0.4816 Accuracy: 0.8597\n",
      "Epoch: 16/160 Train Loss: 0.1878 Accuracy: 0.9344 Time: 10.58291  | Val Loss: 0.3605 Accuracy: 0.8872\n",
      "Epoch: 17/160 Train Loss: 0.1736 Accuracy: 0.9397 Time: 10.62842  | Val Loss: 0.3947 Accuracy: 0.8905\n",
      "Epoch: 18/160 Train Loss: 0.1643 Accuracy: 0.9414 Time: 10.55754  | Val Loss: 0.4301 Accuracy: 0.8744\n",
      "Epoch: 19/160 Train Loss: 0.1483 Accuracy: 0.9474 Time: 10.63794  | Val Loss: 0.3590 Accuracy: 0.8992\n",
      "Epoch: 20/160 Train Loss: 0.1357 Accuracy: 0.9527 Time: 10.62596  | Val Loss: 0.4511 Accuracy: 0.8860\n",
      "Epoch: 21/160 Train Loss: 0.1310 Accuracy: 0.9525 Time: 10.72235  | Val Loss: 0.3862 Accuracy: 0.8888\n",
      "Epoch: 22/160 Train Loss: 0.1265 Accuracy: 0.9552 Time: 10.51055  | Val Loss: 0.4353 Accuracy: 0.8835\n",
      "Epoch: 23/160 Train Loss: 0.1146 Accuracy: 0.9596 Time: 10.40320  | Val Loss: 0.3979 Accuracy: 0.8955\n",
      "Epoch: 24/160 Train Loss: 0.1083 Accuracy: 0.9616 Time: 10.32235  | Val Loss: 0.3678 Accuracy: 0.8981\n",
      "Epoch: 25/160 Train Loss: 0.0997 Accuracy: 0.9651 Time: 10.26564  | Val Loss: 0.4311 Accuracy: 0.8899\n",
      "Epoch: 26/160 Train Loss: 0.1019 Accuracy: 0.9646 Time: 10.30708  | Val Loss: 0.4086 Accuracy: 0.8939\n",
      "Epoch: 27/160 Train Loss: 0.0934 Accuracy: 0.9672 Time: 10.35401  | Val Loss: 0.3840 Accuracy: 0.9012\n",
      "Epoch: 28/160 Train Loss: 0.0860 Accuracy: 0.9698 Time: 10.48244  | Val Loss: 0.4527 Accuracy: 0.8949\n",
      "Epoch: 29/160 Train Loss: 0.0845 Accuracy: 0.9700 Time: 10.29291  | Val Loss: 0.4502 Accuracy: 0.8929\n",
      "Epoch: 30/160 Train Loss: 0.0800 Accuracy: 0.9715 Time: 10.27659  | Val Loss: 0.4014 Accuracy: 0.9013\n",
      "Epoch: 31/160 Train Loss: 0.0791 Accuracy: 0.9724 Time: 10.21120  | Val Loss: 0.4256 Accuracy: 0.9048\n",
      "Epoch: 32/160 Train Loss: 0.0805 Accuracy: 0.9714 Time: 10.16467  | Val Loss: 0.4149 Accuracy: 0.9021\n",
      "Epoch: 33/160 Train Loss: 0.0688 Accuracy: 0.9755 Time: 10.31784  | Val Loss: 0.4584 Accuracy: 0.8948\n",
      "Epoch: 34/160 Train Loss: 0.0682 Accuracy: 0.9759 Time: 10.27032  | Val Loss: 0.4104 Accuracy: 0.9057\n",
      "Epoch: 35/160 Train Loss: 0.0659 Accuracy: 0.9777 Time: 10.17677  | Val Loss: 0.4380 Accuracy: 0.8984\n",
      "Epoch: 36/160 Train Loss: 0.0652 Accuracy: 0.9770 Time: 10.24728  | Val Loss: 0.4414 Accuracy: 0.9090\n",
      "Epoch: 37/160 Train Loss: 0.0636 Accuracy: 0.9780 Time: 10.26133  | Val Loss: 0.4520 Accuracy: 0.9007\n",
      "Epoch: 38/160 Train Loss: 0.0608 Accuracy: 0.9788 Time: 10.28152  | Val Loss: 0.4998 Accuracy: 0.8977\n",
      "Epoch: 39/160 Train Loss: 0.0581 Accuracy: 0.9799 Time: 10.38030  | Val Loss: 0.4133 Accuracy: 0.9079\n",
      "Epoch: 40/160 Train Loss: 0.0530 Accuracy: 0.9821 Time: 10.13551  | Val Loss: 0.4174 Accuracy: 0.9028\n",
      "Epoch: 41/160 Train Loss: 0.0266 Accuracy: 0.9910 Time: 10.09630  | Val Loss: 0.3526 Accuracy: 0.9236\n",
      "Epoch: 42/160 Train Loss: 0.0150 Accuracy: 0.9950 Time: 10.18820  | Val Loss: 0.3631 Accuracy: 0.9254\n",
      "Epoch: 43/160 Train Loss: 0.0115 Accuracy: 0.9963 Time: 10.15897  | Val Loss: 0.3797 Accuracy: 0.9254\n",
      "Epoch: 44/160 Train Loss: 0.0098 Accuracy: 0.9968 Time: 10.28623  | Val Loss: 0.3872 Accuracy: 0.9270\n",
      "Epoch: 45/160 Train Loss: 0.0079 Accuracy: 0.9976 Time: 10.16676  | Val Loss: 0.4011 Accuracy: 0.9253\n",
      "Epoch: 46/160 Train Loss: 0.0070 Accuracy: 0.9981 Time: 10.13052  | Val Loss: 0.4191 Accuracy: 0.9251\n",
      "Epoch: 47/160 Train Loss: 0.0062 Accuracy: 0.9982 Time: 10.29318  | Val Loss: 0.4226 Accuracy: 0.9280\n",
      "Epoch: 48/160 Train Loss: 0.0057 Accuracy: 0.9980 Time: 10.30220  | Val Loss: 0.4326 Accuracy: 0.9273\n",
      "Epoch: 49/160 Train Loss: 0.0050 Accuracy: 0.9984 Time: 10.09299  | Val Loss: 0.4446 Accuracy: 0.9261\n",
      "Epoch: 50/160 Train Loss: 0.0046 Accuracy: 0.9985 Time: 10.20452  | Val Loss: 0.4500 Accuracy: 0.9268\n",
      "Epoch: 51/160 Train Loss: 0.0041 Accuracy: 0.9986 Time: 10.15408  | Val Loss: 0.4630 Accuracy: 0.9282\n",
      "Epoch: 52/160 Train Loss: 0.0041 Accuracy: 0.9988 Time: 10.24109  | Val Loss: 0.4625 Accuracy: 0.9287\n",
      "Epoch: 53/160 Train Loss: 0.0045 Accuracy: 0.9986 Time: 10.44139  | Val Loss: 0.4736 Accuracy: 0.9270\n",
      "Epoch: 54/160 Train Loss: 0.0033 Accuracy: 0.9991 Time: 10.23506  | Val Loss: 0.4776 Accuracy: 0.9285\n",
      "Epoch: 55/160 Train Loss: 0.0037 Accuracy: 0.9989 Time: 10.24590  | Val Loss: 0.4869 Accuracy: 0.9296\n",
      "Epoch: 56/160 Train Loss: 0.0038 Accuracy: 0.9988 Time: 10.25799  | Val Loss: 0.4890 Accuracy: 0.9270\n",
      "Epoch: 57/160 Train Loss: 0.0032 Accuracy: 0.9990 Time: 10.17128  | Val Loss: 0.4957 Accuracy: 0.9284\n",
      "Epoch: 58/160 Train Loss: 0.0032 Accuracy: 0.9989 Time: 10.11667  | Val Loss: 0.5074 Accuracy: 0.9270\n",
      "Epoch: 59/160 Train Loss: 0.0031 Accuracy: 0.9990 Time: 10.07101  | Val Loss: 0.5104 Accuracy: 0.9259\n",
      "Epoch: 60/160 Train Loss: 0.0023 Accuracy: 0.9992 Time: 10.11650  | Val Loss: 0.5199 Accuracy: 0.9269\n",
      "Epoch: 61/160 Train Loss: 0.0037 Accuracy: 0.9987 Time: 10.14791  | Val Loss: 0.5087 Accuracy: 0.9281\n",
      "Epoch: 62/160 Train Loss: 0.0026 Accuracy: 0.9991 Time: 10.05758  | Val Loss: 0.5241 Accuracy: 0.9274\n",
      "Epoch: 63/160 Train Loss: 0.0021 Accuracy: 0.9993 Time: 10.07390  | Val Loss: 0.5299 Accuracy: 0.9283\n",
      "Epoch: 64/160 Train Loss: 0.0029 Accuracy: 0.9991 Time: 10.20663  | Val Loss: 0.5285 Accuracy: 0.9282\n",
      "Epoch: 65/160 Train Loss: 0.0019 Accuracy: 0.9994 Time: 10.11318  | Val Loss: 0.5451 Accuracy: 0.9292\n",
      "Epoch: 66/160 Train Loss: 0.0025 Accuracy: 0.9991 Time: 10.27867  | Val Loss: 0.5522 Accuracy: 0.9266\n",
      "Epoch: 67/160 Train Loss: 0.0027 Accuracy: 0.9992 Time: 10.31245  | Val Loss: 0.5556 Accuracy: 0.9273\n",
      "Epoch: 68/160 Train Loss: 0.0029 Accuracy: 0.9991 Time: 10.47496  | Val Loss: 0.5582 Accuracy: 0.9282\n",
      "Epoch: 69/160 Train Loss: 0.0028 Accuracy: 0.9990 Time: 10.28619  | Val Loss: 0.5481 Accuracy: 0.9283\n",
      "Epoch: 70/160 Train Loss: 0.0020 Accuracy: 0.9994 Time: 10.08710  | Val Loss: 0.5582 Accuracy: 0.9290\n",
      "Epoch: 71/160 Train Loss: 0.0025 Accuracy: 0.9991 Time: 10.29879  | Val Loss: 0.5673 Accuracy: 0.9268\n",
      "Epoch: 72/160 Train Loss: 0.0020 Accuracy: 0.9993 Time: 10.32338  | Val Loss: 0.5580 Accuracy: 0.9279\n",
      "Epoch: 73/160 Train Loss: 0.0023 Accuracy: 0.9992 Time: 10.32630  | Val Loss: 0.5655 Accuracy: 0.9290\n",
      "Epoch: 74/160 Train Loss: 0.0021 Accuracy: 0.9993 Time: 10.30363  | Val Loss: 0.5693 Accuracy: 0.9277\n",
      "Epoch: 75/160 Train Loss: 0.0021 Accuracy: 0.9993 Time: 10.31898  | Val Loss: 0.5757 Accuracy: 0.9281\n",
      "Epoch: 76/160 Train Loss: 0.0016 Accuracy: 0.9995 Time: 10.34965  | Val Loss: 0.5784 Accuracy: 0.9291\n",
      "Epoch: 77/160 Train Loss: 0.0020 Accuracy: 0.9994 Time: 10.06068  | Val Loss: 0.5943 Accuracy: 0.9259\n",
      "Epoch: 78/160 Train Loss: 0.0016 Accuracy: 0.9994 Time: 10.06039  | Val Loss: 0.5849 Accuracy: 0.9273\n",
      "Epoch: 79/160 Train Loss: 0.0020 Accuracy: 0.9993 Time: 10.17279  | Val Loss: 0.5973 Accuracy: 0.9270\n",
      "Epoch: 80/160 Train Loss: 0.0020 Accuracy: 0.9993 Time: 10.14603  | Val Loss: 0.5819 Accuracy: 0.9279\n",
      "Epoch: 81/160 Train Loss: 0.0016 Accuracy: 0.9995 Time: 10.29316  | Val Loss: 0.5754 Accuracy: 0.9279\n",
      "Epoch: 82/160 Train Loss: 0.0016 Accuracy: 0.9994 Time: 10.24664  | Val Loss: 0.5762 Accuracy: 0.9289\n",
      "Epoch: 83/160 Train Loss: 0.0013 Accuracy: 0.9996 Time: 10.21921  | Val Loss: 0.5733 Accuracy: 0.9297\n",
      "Epoch: 84/160 Train Loss: 0.0017 Accuracy: 0.9995 Time: 10.31386  | Val Loss: 0.5753 Accuracy: 0.9298\n",
      "Epoch: 85/160 Train Loss: 0.0012 Accuracy: 0.9997 Time: 10.34885  | Val Loss: 0.5730 Accuracy: 0.9295\n",
      "Epoch: 86/160 Train Loss: 0.0011 Accuracy: 0.9997 Time: 10.31245  | Val Loss: 0.5653 Accuracy: 0.9298\n",
      "Epoch: 87/160 Train Loss: 0.0010 Accuracy: 0.9997 Time: 10.11321  | Val Loss: 0.5747 Accuracy: 0.9289\n",
      "Epoch: 88/160 Train Loss: 0.0010 Accuracy: 0.9998 Time: 10.25772  | Val Loss: 0.5669 Accuracy: 0.9291\n",
      "Epoch: 89/160 Train Loss: 0.0011 Accuracy: 0.9996 Time: 10.34667  | Val Loss: 0.5658 Accuracy: 0.9303\n",
      "Epoch: 90/160 Train Loss: 0.0011 Accuracy: 0.9996 Time: 10.17101  | Val Loss: 0.5712 Accuracy: 0.9295\n",
      "Epoch: 91/160 Train Loss: 0.0008 Accuracy: 0.9997 Time: 10.33086  | Val Loss: 0.5594 Accuracy: 0.9303\n",
      "Epoch: 92/160 Train Loss: 0.0008 Accuracy: 0.9998 Time: 10.27308  | Val Loss: 0.5698 Accuracy: 0.9299\n",
      "Epoch: 93/160 Train Loss: 0.0009 Accuracy: 0.9997 Time: 10.21956  | Val Loss: 0.5703 Accuracy: 0.9298\n",
      "Epoch: 94/160 Train Loss: 0.0009 Accuracy: 0.9997 Time: 10.34234  | Val Loss: 0.5655 Accuracy: 0.9300\n",
      "Epoch: 95/160 Train Loss: 0.0005 Accuracy: 0.9999 Time: 10.06332  | Val Loss: 0.5648 Accuracy: 0.9304\n",
      "Epoch: 96/160 Train Loss: 0.0006 Accuracy: 0.9998 Time: 10.31553  | Val Loss: 0.5660 Accuracy: 0.9299\n",
      "Epoch: 97/160 Train Loss: 0.0007 Accuracy: 0.9998 Time: 10.30816  | Val Loss: 0.5689 Accuracy: 0.9312\n",
      "Epoch: 98/160 Train Loss: 0.0007 Accuracy: 0.9998 Time: 10.20660  | Val Loss: 0.5630 Accuracy: 0.9303\n",
      "Epoch: 99/160 Train Loss: 0.0008 Accuracy: 0.9998 Time: 10.13444  | Val Loss: 0.5632 Accuracy: 0.9307\n",
      "Epoch: 100/160 Train Loss: 0.0007 Accuracy: 0.9997 Time: 10.17976  | Val Loss: 0.5692 Accuracy: 0.9310\n",
      "Epoch: 101/160 Train Loss: 0.0006 Accuracy: 0.9999 Time: 10.28018  | Val Loss: 0.5683 Accuracy: 0.9314\n",
      "Epoch: 102/160 Train Loss: 0.0005 Accuracy: 0.9999 Time: 10.14178  | Val Loss: 0.5688 Accuracy: 0.9314\n",
      "Epoch: 103/160 Train Loss: 0.0007 Accuracy: 0.9998 Time: 10.15738  | Val Loss: 0.5688 Accuracy: 0.9305\n",
      "Epoch: 104/160 Train Loss: 0.0004 Accuracy: 1.0000 Time: 10.09707  | Val Loss: 0.5718 Accuracy: 0.9305\n",
      "Epoch: 105/160 Train Loss: 0.0009 Accuracy: 0.9997 Time: 10.29579  | Val Loss: 0.5705 Accuracy: 0.9303\n",
      "Epoch: 106/160 Train Loss: 0.0006 Accuracy: 0.9998 Time: 10.26626  | Val Loss: 0.5730 Accuracy: 0.9298\n",
      "Epoch: 107/160 Train Loss: 0.0005 Accuracy: 0.9999 Time: 10.23657  | Val Loss: 0.5675 Accuracy: 0.9315\n",
      "Epoch: 108/160 Train Loss: 0.0005 Accuracy: 0.9998 Time: 10.36600  | Val Loss: 0.5695 Accuracy: 0.9306\n",
      "Epoch: 109/160 Train Loss: 0.0005 Accuracy: 0.9999 Time: 10.19732  | Val Loss: 0.5654 Accuracy: 0.9304\n",
      "Epoch: 110/160 Train Loss: 0.0007 Accuracy: 0.9998 Time: 10.22197  | Val Loss: 0.5649 Accuracy: 0.9307\n",
      "Epoch: 111/160 Train Loss: 0.0006 Accuracy: 0.9998 Time: 10.17663  | Val Loss: 0.5713 Accuracy: 0.9310\n",
      "Epoch: 112/160 Train Loss: 0.0007 Accuracy: 0.9998 Time: 10.32531  | Val Loss: 0.5724 Accuracy: 0.9320\n",
      "Epoch: 113/160 Train Loss: 0.0005 Accuracy: 0.9999 Time: 10.27449  | Val Loss: 0.5726 Accuracy: 0.9312\n",
      "Epoch: 114/160 Train Loss: 0.0007 Accuracy: 0.9998 Time: 10.26046  | Val Loss: 0.5743 Accuracy: 0.9304\n",
      "Epoch: 115/160 Train Loss: 0.0006 Accuracy: 0.9998 Time: 10.28330  | Val Loss: 0.5767 Accuracy: 0.9314\n",
      "Epoch: 116/160 Train Loss: 0.0005 Accuracy: 0.9999 Time: 10.07267  | Val Loss: 0.5720 Accuracy: 0.9314\n",
      "Epoch: 117/160 Train Loss: 0.0005 Accuracy: 0.9999 Time: 10.10458  | Val Loss: 0.5761 Accuracy: 0.9323\n",
      "Epoch: 118/160 Train Loss: 0.0005 Accuracy: 0.9998 Time: 10.24409  | Val Loss: 0.5765 Accuracy: 0.9313\n",
      "Epoch: 119/160 Train Loss: 0.0005 Accuracy: 0.9999 Time: 10.10193  | Val Loss: 0.5735 Accuracy: 0.9312\n",
      "Epoch: 120/160 Train Loss: 0.0004 Accuracy: 0.9999 Time: 10.07688  | Val Loss: 0.5803 Accuracy: 0.9320\n",
      "Epoch: 121/160 Train Loss: 0.0004 Accuracy: 0.9999 Time: 10.13875  | Val Loss: 0.5805 Accuracy: 0.9299\n",
      "Epoch: 122/160 Train Loss: 0.0005 Accuracy: 0.9999 Time: 10.11659  | Val Loss: 0.5790 Accuracy: 0.9318\n",
      "Epoch: 123/160 Train Loss: 0.0006 Accuracy: 0.9999 Time: 10.18305  | Val Loss: 0.5795 Accuracy: 0.9311\n",
      "Epoch: 124/160 Train Loss: 0.0006 Accuracy: 0.9998 Time: 10.17212  | Val Loss: 0.5820 Accuracy: 0.9311\n",
      "Epoch: 125/160 Train Loss: 0.0004 Accuracy: 1.0000 Time: 10.23539  | Val Loss: 0.5787 Accuracy: 0.9321\n",
      "Epoch: 126/160 Train Loss: 0.0006 Accuracy: 0.9998 Time: 10.33121  | Val Loss: 0.5759 Accuracy: 0.9318\n",
      "Epoch: 127/160 Train Loss: 0.0005 Accuracy: 0.9998 Time: 10.25337  | Val Loss: 0.5761 Accuracy: 0.9318\n",
      "Epoch: 128/160 Train Loss: 0.0006 Accuracy: 0.9998 Time: 10.23221  | Val Loss: 0.5770 Accuracy: 0.9317\n",
      "Epoch: 129/160 Train Loss: 0.0003 Accuracy: 0.9999 Time: 10.24411  | Val Loss: 0.5778 Accuracy: 0.9311\n",
      "Epoch: 130/160 Train Loss: 0.0005 Accuracy: 0.9999 Time: 10.27923  | Val Loss: 0.5808 Accuracy: 0.9315\n",
      "Epoch: 131/160 Train Loss: 0.0006 Accuracy: 0.9998 Time: 10.09717  | Val Loss: 0.5761 Accuracy: 0.9318\n",
      "Epoch: 132/160 Train Loss: 0.0004 Accuracy: 0.9999 Time: 10.09550  | Val Loss: 0.5760 Accuracy: 0.9327\n",
      "Epoch: 133/160 Train Loss: 0.0004 Accuracy: 0.9999 Time: 10.11449  | Val Loss: 0.5787 Accuracy: 0.9308\n",
      "Epoch: 134/160 Train Loss: 0.0006 Accuracy: 0.9998 Time: 10.13077  | Val Loss: 0.5792 Accuracy: 0.9314\n",
      "Epoch: 135/160 Train Loss: 0.0004 Accuracy: 0.9998 Time: 10.21844  | Val Loss: 0.5744 Accuracy: 0.9323\n",
      "Epoch: 136/160 Train Loss: 0.0004 Accuracy: 0.9999 Time: 10.08820  | Val Loss: 0.5787 Accuracy: 0.9317\n",
      "Epoch: 137/160 Train Loss: 0.0003 Accuracy: 0.9999 Time: 10.34653  | Val Loss: 0.5817 Accuracy: 0.9309\n",
      "Epoch: 138/160 Train Loss: 0.0006 Accuracy: 0.9998 Time: 10.38075  | Val Loss: 0.5754 Accuracy: 0.9318\n",
      "Epoch: 139/160 Train Loss: 0.0006 Accuracy: 0.9999 Time: 10.17417  | Val Loss: 0.5828 Accuracy: 0.9311\n",
      "Epoch: 140/160 Train Loss: 0.0005 Accuracy: 0.9999 Time: 10.28257  | Val Loss: 0.5787 Accuracy: 0.9319\n",
      "Epoch: 141/160 Train Loss: 0.0008 Accuracy: 0.9998 Time: 10.25249  | Val Loss: 0.5801 Accuracy: 0.9303\n",
      "Epoch: 142/160 Train Loss: 0.0005 Accuracy: 0.9998 Time: 10.26949  | Val Loss: 0.5823 Accuracy: 0.9311\n",
      "Epoch: 143/160 Train Loss: 0.0004 Accuracy: 0.9999 Time: 10.28947  | Val Loss: 0.5794 Accuracy: 0.9321\n",
      "Epoch: 144/160 Train Loss: 0.0005 Accuracy: 0.9999 Time: 10.28480  | Val Loss: 0.5746 Accuracy: 0.9321\n",
      "Epoch: 145/160 Train Loss: 0.0004 Accuracy: 0.9999 Time: 10.22049  | Val Loss: 0.5769 Accuracy: 0.9322\n",
      "Epoch: 146/160 Train Loss: 0.0004 Accuracy: 0.9999 Time: 10.24025  | Val Loss: 0.5779 Accuracy: 0.9320\n",
      "Epoch: 147/160 Train Loss: 0.0006 Accuracy: 0.9999 Time: 10.26447  | Val Loss: 0.5770 Accuracy: 0.9317\n",
      "Epoch: 148/160 Train Loss: 0.0003 Accuracy: 0.9999 Time: 10.21805  | Val Loss: 0.5766 Accuracy: 0.9313\n",
      "Epoch: 149/160 Train Loss: 0.0004 Accuracy: 0.9999 Time: 10.29191  | Val Loss: 0.5775 Accuracy: 0.9321\n",
      "Epoch: 150/160 Train Loss: 0.0004 Accuracy: 0.9999 Time: 10.12568  | Val Loss: 0.5787 Accuracy: 0.9314\n",
      "Epoch: 151/160 Train Loss: 0.0003 Accuracy: 0.9999 Time: 10.17485  | Val Loss: 0.5741 Accuracy: 0.9312\n",
      "Epoch: 152/160 Train Loss: 0.0006 Accuracy: 0.9999 Time: 10.36413  | Val Loss: 0.5813 Accuracy: 0.9318\n",
      "Epoch: 153/160 Train Loss: 0.0005 Accuracy: 0.9999 Time: 10.34463  | Val Loss: 0.5765 Accuracy: 0.9320\n",
      "Epoch: 154/160 Train Loss: 0.0004 Accuracy: 0.9999 Time: 10.22929  | Val Loss: 0.5758 Accuracy: 0.9318\n",
      "Epoch: 155/160 Train Loss: 0.0005 Accuracy: 0.9999 Time: 10.22908  | Val Loss: 0.5741 Accuracy: 0.9319\n",
      "Epoch: 156/160 Train Loss: 0.0005 Accuracy: 0.9999 Time: 10.05800  | Val Loss: 0.5753 Accuracy: 0.9313\n",
      "Epoch: 157/160 Train Loss: 0.0005 Accuracy: 0.9999 Time: 10.26287  | Val Loss: 0.5769 Accuracy: 0.9320\n",
      "Epoch: 158/160 Train Loss: 0.0004 Accuracy: 0.9998 Time: 10.25728  | Val Loss: 0.5745 Accuracy: 0.9322\n",
      "Epoch: 159/160 Train Loss: 0.0005 Accuracy: 0.9999 Time: 10.22555  | Val Loss: 0.5784 Accuracy: 0.9308\n",
      "Epoch: 160/160 Train Loss: 0.0006 Accuracy: 0.9999 Time: 10.35156  | Val Loss: 0.5778 Accuracy: 0.9314\n",
      "#Parameter: 21282122 Accuracy: 0.9314\n"
     ]
    }
   ],
   "source": [
    "from hdd.models.cnn.resnet import ResnetSmall, resnet34_config\n",
    "from hdd.train.classification_utils import (\n",
    "    naive_train_classification_model,\n",
    "    eval_image_classifier,\n",
    ")\n",
    "from hdd.models.nn_utils import count_trainable_parameter\n",
    "\n",
    "\n",
    "def train_net(\n",
    "    resnet_config,\n",
    "    train_dataloader,\n",
    "    val_dataloader,\n",
    "    dropout,\n",
    "    lr,\n",
    "    weight_decay,\n",
    "    step_size=40,\n",
    "    gamma=0.1,\n",
    "    max_epochs=160,\n",
    ") -> tuple[ResnetSmall, dict[str, list[float]]]:\n",
    "    net = ResnetSmall(resnet_config, num_classes=10, dropout=dropout).to(DEVICE)\n",
    "    criteria = nn.CrossEntropyLoss()\n",
    "    # SGD的收敛速度远不如Adam好\n",
    "    # optimizer = torch.optim.SGD(\n",
    "    #     net.parameters(), lr=lr, momentum=0.9, weight_decay=weight_decay\n",
    "    # )\n",
    "    optimizer = optim.AdamW(\n",
    "        net.parameters(), lr=lr, eps=1e-6, weight_decay=weight_decay\n",
    "    )\n",
    "    scheduler = torch.optim.lr_scheduler.StepLR(\n",
    "        optimizer, step_size=step_size, gamma=gamma, last_epoch=-1\n",
    "    )\n",
    "    training_stats = naive_train_classification_model(\n",
    "        net,\n",
    "        criteria,\n",
    "        max_epochs,\n",
    "        train_dataloader,\n",
    "        val_dataloader,\n",
    "        DEVICE,\n",
    "        optimizer,\n",
    "        scheduler,\n",
    "        verbose=True,\n",
    "    )\n",
    "    return net, training_stats\n",
    "\n",
    "\n",
    "net, _ = train_net(\n",
    "    resnet34_config,\n",
    "    train_dataloader,\n",
    "    val_dataloader,\n",
    "    dropout=0.0,\n",
    "    lr=0.01,\n",
    "    weight_decay=0,\n",
    ")\n",
    "\n",
    "eval_result = eval_image_classifier(net, val_dataloader.dataset, DEVICE)\n",
    "ss = [result.gt_label == result.predicted_label for result in eval_result]\n",
    "print(f\"#Parameter: {count_trainable_parameter(net)} Accuracy: {sum(ss) / len(ss)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5a79f386",
   "metadata": {},
   "source": [
    "### [AutoAugmentation](https://arxiv.org/abs/1805.09501v1)\n",
    "\n",
    "```python\n",
    "transforms.RandomCrop(size=32, padding=4),\n",
    "transforms.RandomHorizontalFlip(),\n",
    "CIFAR10Policy(),\n",
    "transforms.ToTensor(),\n",
    "transforms.Normalize(mean=TRAIN_MEAN, std=TRAIN_STD),\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9b098775",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n",
      "Epoch: 1/160 Train Loss: 1.8869 Accuracy: 0.2928 Time: 10.49294  | Val Loss: 1.4959 Accuracy: 0.4509\n",
      "Epoch: 2/160 Train Loss: 1.3630 Accuracy: 0.5111 Time: 10.45894  | Val Loss: 1.0653 Accuracy: 0.6243\n",
      "Epoch: 3/160 Train Loss: 1.1004 Accuracy: 0.6115 Time: 10.51756  | Val Loss: 0.8691 Accuracy: 0.7004\n",
      "Epoch: 4/160 Train Loss: 0.9250 Accuracy: 0.6752 Time: 10.50109  | Val Loss: 0.7017 Accuracy: 0.7632\n",
      "Epoch: 5/160 Train Loss: 0.8099 Accuracy: 0.7151 Time: 10.45488  | Val Loss: 0.6079 Accuracy: 0.7844\n",
      "Epoch: 6/160 Train Loss: 0.7292 Accuracy: 0.7440 Time: 10.71078  | Val Loss: 0.5311 Accuracy: 0.8152\n",
      "Epoch: 7/160 Train Loss: 0.6726 Accuracy: 0.7669 Time: 10.55842  | Val Loss: 0.5624 Accuracy: 0.8097\n",
      "Epoch: 8/160 Train Loss: 0.6267 Accuracy: 0.7826 Time: 10.49996  | Val Loss: 0.5189 Accuracy: 0.8199\n",
      "Epoch: 9/160 Train Loss: 0.5896 Accuracy: 0.7939 Time: 10.54540  | Val Loss: 0.4596 Accuracy: 0.8390\n",
      "Epoch: 10/160 Train Loss: 0.5595 Accuracy: 0.8055 Time: 10.59277  | Val Loss: 0.4427 Accuracy: 0.8466\n",
      "Epoch: 11/160 Train Loss: 0.5268 Accuracy: 0.8149 Time: 10.53884  | Val Loss: 0.4232 Accuracy: 0.8567\n",
      "Epoch: 12/160 Train Loss: 0.5049 Accuracy: 0.8243 Time: 10.50386  | Val Loss: 0.3959 Accuracy: 0.8626\n",
      "Epoch: 13/160 Train Loss: 0.4834 Accuracy: 0.8321 Time: 10.56814  | Val Loss: 0.3840 Accuracy: 0.8692\n",
      "Epoch: 14/160 Train Loss: 0.4580 Accuracy: 0.8411 Time: 10.65456  | Val Loss: 0.3477 Accuracy: 0.8845\n",
      "Epoch: 15/160 Train Loss: 0.4416 Accuracy: 0.8462 Time: 10.53508  | Val Loss: 0.3654 Accuracy: 0.8777\n",
      "Epoch: 16/160 Train Loss: 0.4222 Accuracy: 0.8519 Time: 10.50095  | Val Loss: 0.3948 Accuracy: 0.8702\n",
      "Epoch: 17/160 Train Loss: 0.4087 Accuracy: 0.8561 Time: 10.51043  | Val Loss: 0.3817 Accuracy: 0.8791\n",
      "Epoch: 18/160 Train Loss: 0.3976 Accuracy: 0.8623 Time: 10.54038  | Val Loss: 0.3212 Accuracy: 0.8941\n",
      "Epoch: 19/160 Train Loss: 0.3781 Accuracy: 0.8680 Time: 10.49531  | Val Loss: 0.3172 Accuracy: 0.8979\n",
      "Epoch: 20/160 Train Loss: 0.3694 Accuracy: 0.8701 Time: 10.47330  | Val Loss: 0.3347 Accuracy: 0.8878\n",
      "Epoch: 21/160 Train Loss: 0.3606 Accuracy: 0.8749 Time: 10.43808  | Val Loss: 0.3372 Accuracy: 0.8882\n",
      "Epoch: 22/160 Train Loss: 0.3467 Accuracy: 0.8789 Time: 10.38942  | Val Loss: 0.3137 Accuracy: 0.8992\n",
      "Epoch: 23/160 Train Loss: 0.3381 Accuracy: 0.8830 Time: 10.36830  | Val Loss: 0.2908 Accuracy: 0.9060\n",
      "Epoch: 24/160 Train Loss: 0.3275 Accuracy: 0.8860 Time: 10.43790  | Val Loss: 0.3252 Accuracy: 0.8978\n",
      "Epoch: 25/160 Train Loss: 0.3239 Accuracy: 0.8870 Time: 10.50415  | Val Loss: 0.2922 Accuracy: 0.9068\n",
      "Epoch: 26/160 Train Loss: 0.3156 Accuracy: 0.8879 Time: 10.57479  | Val Loss: 0.3094 Accuracy: 0.9013\n",
      "Epoch: 27/160 Train Loss: 0.3000 Accuracy: 0.8964 Time: 10.55743  | Val Loss: 0.3013 Accuracy: 0.9052\n",
      "Epoch: 28/160 Train Loss: 0.2946 Accuracy: 0.8977 Time: 10.56115  | Val Loss: 0.3141 Accuracy: 0.8981\n",
      "Epoch: 29/160 Train Loss: 0.2954 Accuracy: 0.8966 Time: 10.57919  | Val Loss: 0.2883 Accuracy: 0.9080\n",
      "Epoch: 30/160 Train Loss: 0.2885 Accuracy: 0.8991 Time: 10.60959  | Val Loss: 0.2919 Accuracy: 0.9088\n",
      "Epoch: 31/160 Train Loss: 0.2732 Accuracy: 0.9052 Time: 10.51522  | Val Loss: 0.2918 Accuracy: 0.9128\n",
      "Epoch: 32/160 Train Loss: 0.2729 Accuracy: 0.9044 Time: 10.44515  | Val Loss: 0.2934 Accuracy: 0.9104\n",
      "Epoch: 33/160 Train Loss: 0.2679 Accuracy: 0.9063 Time: 10.32741  | Val Loss: 0.2685 Accuracy: 0.9184\n",
      "Epoch: 34/160 Train Loss: 0.2674 Accuracy: 0.9058 Time: 10.19755  | Val Loss: 0.3091 Accuracy: 0.9071\n",
      "Epoch: 35/160 Train Loss: 0.2603 Accuracy: 0.9089 Time: 10.17996  | Val Loss: 0.2799 Accuracy: 0.9140\n",
      "Epoch: 36/160 Train Loss: 0.2547 Accuracy: 0.9118 Time: 10.17831  | Val Loss: 0.3265 Accuracy: 0.9099\n",
      "Epoch: 37/160 Train Loss: 0.2437 Accuracy: 0.9145 Time: 10.11238  | Val Loss: 0.2799 Accuracy: 0.9167\n",
      "Epoch: 38/160 Train Loss: 0.2394 Accuracy: 0.9162 Time: 10.12712  | Val Loss: 0.2811 Accuracy: 0.9173\n",
      "Epoch: 39/160 Train Loss: 0.2373 Accuracy: 0.9178 Time: 10.16291  | Val Loss: 0.2617 Accuracy: 0.9173\n",
      "Epoch: 40/160 Train Loss: 0.2342 Accuracy: 0.9184 Time: 10.18743  | Val Loss: 0.2988 Accuracy: 0.9180\n",
      "Epoch: 41/160 Train Loss: 0.1914 Accuracy: 0.9329 Time: 10.21075  | Val Loss: 0.2318 Accuracy: 0.9362\n",
      "Epoch: 42/160 Train Loss: 0.1738 Accuracy: 0.9391 Time: 10.15715  | Val Loss: 0.2277 Accuracy: 0.9375\n",
      "Epoch: 43/160 Train Loss: 0.1639 Accuracy: 0.9420 Time: 10.14680  | Val Loss: 0.2308 Accuracy: 0.9370\n",
      "Epoch: 44/160 Train Loss: 0.1596 Accuracy: 0.9444 Time: 10.14872  | Val Loss: 0.2230 Accuracy: 0.9397\n",
      "Epoch: 45/160 Train Loss: 0.1500 Accuracy: 0.9477 Time: 10.28621  | Val Loss: 0.2237 Accuracy: 0.9400\n",
      "Epoch: 46/160 Train Loss: 0.1519 Accuracy: 0.9483 Time: 10.20779  | Val Loss: 0.2251 Accuracy: 0.9388\n",
      "Epoch: 47/160 Train Loss: 0.1514 Accuracy: 0.9469 Time: 10.18410  | Val Loss: 0.2231 Accuracy: 0.9398\n",
      "Epoch: 48/160 Train Loss: 0.1438 Accuracy: 0.9500 Time: 10.20999  | Val Loss: 0.2284 Accuracy: 0.9399\n",
      "Epoch: 49/160 Train Loss: 0.1447 Accuracy: 0.9504 Time: 10.21732  | Val Loss: 0.2265 Accuracy: 0.9399\n",
      "Epoch: 50/160 Train Loss: 0.1421 Accuracy: 0.9502 Time: 10.21769  | Val Loss: 0.2262 Accuracy: 0.9410\n",
      "Epoch: 51/160 Train Loss: 0.1441 Accuracy: 0.9493 Time: 10.18796  | Val Loss: 0.2269 Accuracy: 0.9410\n",
      "Epoch: 52/160 Train Loss: 0.1361 Accuracy: 0.9531 Time: 10.19253  | Val Loss: 0.2186 Accuracy: 0.9426\n",
      "Epoch: 53/160 Train Loss: 0.1319 Accuracy: 0.9537 Time: 10.35754  | Val Loss: 0.2210 Accuracy: 0.9408\n",
      "Epoch: 54/160 Train Loss: 0.1361 Accuracy: 0.9521 Time: 10.25233  | Val Loss: 0.2304 Accuracy: 0.9404\n",
      "Epoch: 55/160 Train Loss: 0.1317 Accuracy: 0.9542 Time: 10.20967  | Val Loss: 0.2289 Accuracy: 0.9408\n",
      "Epoch: 56/160 Train Loss: 0.1310 Accuracy: 0.9541 Time: 10.18761  | Val Loss: 0.2257 Accuracy: 0.9426\n",
      "Epoch: 57/160 Train Loss: 0.1310 Accuracy: 0.9539 Time: 10.20172  | Val Loss: 0.2300 Accuracy: 0.9398\n",
      "Epoch: 58/160 Train Loss: 0.1296 Accuracy: 0.9544 Time: 10.12693  | Val Loss: 0.2266 Accuracy: 0.9421\n",
      "Epoch: 59/160 Train Loss: 0.1267 Accuracy: 0.9559 Time: 10.09800  | Val Loss: 0.2309 Accuracy: 0.9430\n",
      "Epoch: 60/160 Train Loss: 0.1235 Accuracy: 0.9564 Time: 10.24898  | Val Loss: 0.2333 Accuracy: 0.9414\n",
      "Epoch: 61/160 Train Loss: 0.1287 Accuracy: 0.9563 Time: 10.14072  | Val Loss: 0.2293 Accuracy: 0.9415\n",
      "Epoch: 62/160 Train Loss: 0.1249 Accuracy: 0.9567 Time: 10.22934  | Val Loss: 0.2296 Accuracy: 0.9416\n",
      "Epoch: 63/160 Train Loss: 0.1279 Accuracy: 0.9557 Time: 10.16995  | Val Loss: 0.2331 Accuracy: 0.9418\n",
      "Epoch: 64/160 Train Loss: 0.1287 Accuracy: 0.9558 Time: 10.17791  | Val Loss: 0.2298 Accuracy: 0.9422\n",
      "Epoch: 65/160 Train Loss: 0.1249 Accuracy: 0.9567 Time: 10.20522  | Val Loss: 0.2340 Accuracy: 0.9410\n",
      "Epoch: 66/160 Train Loss: 0.1184 Accuracy: 0.9591 Time: 10.18091  | Val Loss: 0.2412 Accuracy: 0.9413\n",
      "Epoch: 67/160 Train Loss: 0.1237 Accuracy: 0.9568 Time: 10.13973  | Val Loss: 0.2367 Accuracy: 0.9409\n",
      "Epoch: 68/160 Train Loss: 0.1207 Accuracy: 0.9588 Time: 10.18871  | Val Loss: 0.2373 Accuracy: 0.9402\n",
      "Epoch: 69/160 Train Loss: 0.1224 Accuracy: 0.9585 Time: 10.31193  | Val Loss: 0.2349 Accuracy: 0.9422\n",
      "Epoch: 70/160 Train Loss: 0.1168 Accuracy: 0.9596 Time: 10.23655  | Val Loss: 0.2378 Accuracy: 0.9420\n",
      "Epoch: 71/160 Train Loss: 0.1204 Accuracy: 0.9590 Time: 10.19863  | Val Loss: 0.2368 Accuracy: 0.9409\n",
      "Epoch: 72/160 Train Loss: 0.1151 Accuracy: 0.9599 Time: 10.16729  | Val Loss: 0.2386 Accuracy: 0.9410\n",
      "Epoch: 73/160 Train Loss: 0.1166 Accuracy: 0.9591 Time: 10.14242  | Val Loss: 0.2368 Accuracy: 0.9420\n",
      "Epoch: 74/160 Train Loss: 0.1180 Accuracy: 0.9586 Time: 10.25754  | Val Loss: 0.2397 Accuracy: 0.9420\n",
      "Epoch: 75/160 Train Loss: 0.1168 Accuracy: 0.9589 Time: 10.29734  | Val Loss: 0.2464 Accuracy: 0.9405\n",
      "Epoch: 76/160 Train Loss: 0.1155 Accuracy: 0.9591 Time: 10.23075  | Val Loss: 0.2380 Accuracy: 0.9409\n",
      "Epoch: 77/160 Train Loss: 0.1166 Accuracy: 0.9605 Time: 10.08420  | Val Loss: 0.2370 Accuracy: 0.9419\n",
      "Epoch: 78/160 Train Loss: 0.1135 Accuracy: 0.9609 Time: 10.09851  | Val Loss: 0.2407 Accuracy: 0.9420\n",
      "Epoch: 79/160 Train Loss: 0.1156 Accuracy: 0.9605 Time: 10.21920  | Val Loss: 0.2436 Accuracy: 0.9416\n",
      "Epoch: 80/160 Train Loss: 0.1147 Accuracy: 0.9600 Time: 10.16860  | Val Loss: 0.2381 Accuracy: 0.9419\n",
      "Epoch: 81/160 Train Loss: 0.1111 Accuracy: 0.9612 Time: 10.18821  | Val Loss: 0.2349 Accuracy: 0.9429\n",
      "Epoch: 82/160 Train Loss: 0.1128 Accuracy: 0.9605 Time: 10.25776  | Val Loss: 0.2348 Accuracy: 0.9427\n",
      "Epoch: 83/160 Train Loss: 0.1109 Accuracy: 0.9614 Time: 10.33345  | Val Loss: 0.2375 Accuracy: 0.9431\n",
      "Epoch: 84/160 Train Loss: 0.1079 Accuracy: 0.9627 Time: 10.14827  | Val Loss: 0.2361 Accuracy: 0.9434\n",
      "Epoch: 85/160 Train Loss: 0.1080 Accuracy: 0.9628 Time: 10.24028  | Val Loss: 0.2352 Accuracy: 0.9431\n",
      "Epoch: 86/160 Train Loss: 0.1088 Accuracy: 0.9620 Time: 10.19857  | Val Loss: 0.2307 Accuracy: 0.9437\n",
      "Epoch: 87/160 Train Loss: 0.1079 Accuracy: 0.9630 Time: 10.24357  | Val Loss: 0.2340 Accuracy: 0.9442\n",
      "Epoch: 88/160 Train Loss: 0.1091 Accuracy: 0.9622 Time: 10.20938  | Val Loss: 0.2335 Accuracy: 0.9438\n",
      "Epoch: 89/160 Train Loss: 0.1095 Accuracy: 0.9617 Time: 10.18877  | Val Loss: 0.2326 Accuracy: 0.9439\n",
      "Epoch: 90/160 Train Loss: 0.1064 Accuracy: 0.9629 Time: 10.16312  | Val Loss: 0.2331 Accuracy: 0.9438\n",
      "Epoch: 91/160 Train Loss: 0.1061 Accuracy: 0.9629 Time: 10.13766  | Val Loss: 0.2338 Accuracy: 0.9439\n",
      "Epoch: 92/160 Train Loss: 0.1075 Accuracy: 0.9634 Time: 10.20389  | Val Loss: 0.2348 Accuracy: 0.9439\n",
      "Epoch: 93/160 Train Loss: 0.1059 Accuracy: 0.9626 Time: 10.20483  | Val Loss: 0.2344 Accuracy: 0.9428\n",
      "Epoch: 94/160 Train Loss: 0.1076 Accuracy: 0.9624 Time: 10.28102  | Val Loss: 0.2333 Accuracy: 0.9426\n",
      "Epoch: 95/160 Train Loss: 0.1060 Accuracy: 0.9631 Time: 10.25890  | Val Loss: 0.2359 Accuracy: 0.9431\n",
      "Epoch: 96/160 Train Loss: 0.1072 Accuracy: 0.9629 Time: 10.18451  | Val Loss: 0.2367 Accuracy: 0.9426\n",
      "Epoch: 97/160 Train Loss: 0.1063 Accuracy: 0.9637 Time: 10.24417  | Val Loss: 0.2359 Accuracy: 0.9436\n",
      "Epoch: 98/160 Train Loss: 0.1055 Accuracy: 0.9631 Time: 10.24226  | Val Loss: 0.2343 Accuracy: 0.9437\n",
      "Epoch: 99/160 Train Loss: 0.1066 Accuracy: 0.9624 Time: 10.20135  | Val Loss: 0.2359 Accuracy: 0.9430\n",
      "Epoch: 100/160 Train Loss: 0.1098 Accuracy: 0.9625 Time: 10.21465  | Val Loss: 0.2336 Accuracy: 0.9431\n",
      "Epoch: 101/160 Train Loss: 0.1105 Accuracy: 0.9617 Time: 10.13355  | Val Loss: 0.2328 Accuracy: 0.9442\n",
      "Epoch: 102/160 Train Loss: 0.1073 Accuracy: 0.9626 Time: 10.22126  | Val Loss: 0.2311 Accuracy: 0.9449\n",
      "Epoch: 103/160 Train Loss: 0.1041 Accuracy: 0.9642 Time: 10.26644  | Val Loss: 0.2344 Accuracy: 0.9436\n",
      "Epoch: 104/160 Train Loss: 0.1070 Accuracy: 0.9626 Time: 10.33414  | Val Loss: 0.2349 Accuracy: 0.9430\n",
      "Epoch: 105/160 Train Loss: 0.1051 Accuracy: 0.9628 Time: 10.31547  | Val Loss: 0.2331 Accuracy: 0.9444\n",
      "Epoch: 106/160 Train Loss: 0.1076 Accuracy: 0.9628 Time: 10.24985  | Val Loss: 0.2332 Accuracy: 0.9446\n",
      "Epoch: 107/160 Train Loss: 0.1086 Accuracy: 0.9622 Time: 10.23641  | Val Loss: 0.2347 Accuracy: 0.9435\n",
      "Epoch: 108/160 Train Loss: 0.1063 Accuracy: 0.9627 Time: 10.13388  | Val Loss: 0.2332 Accuracy: 0.9425\n",
      "Epoch: 109/160 Train Loss: 0.1087 Accuracy: 0.9621 Time: 10.20195  | Val Loss: 0.2335 Accuracy: 0.9430\n",
      "Epoch: 110/160 Train Loss: 0.1019 Accuracy: 0.9640 Time: 10.18489  | Val Loss: 0.2384 Accuracy: 0.9427\n",
      "Epoch: 111/160 Train Loss: 0.1018 Accuracy: 0.9647 Time: 10.19785  | Val Loss: 0.2328 Accuracy: 0.9435\n",
      "Epoch: 112/160 Train Loss: 0.1029 Accuracy: 0.9646 Time: 10.18763  | Val Loss: 0.2340 Accuracy: 0.9431\n",
      "Epoch: 113/160 Train Loss: 0.1074 Accuracy: 0.9629 Time: 10.20337  | Val Loss: 0.2357 Accuracy: 0.9436\n",
      "Epoch: 114/160 Train Loss: 0.1018 Accuracy: 0.9638 Time: 10.14242  | Val Loss: 0.2372 Accuracy: 0.9437\n",
      "Epoch: 115/160 Train Loss: 0.1036 Accuracy: 0.9642 Time: 10.15171  | Val Loss: 0.2356 Accuracy: 0.9433\n",
      "Epoch: 116/160 Train Loss: 0.1024 Accuracy: 0.9641 Time: 10.11061  | Val Loss: 0.2382 Accuracy: 0.9434\n",
      "Epoch: 117/160 Train Loss: 0.1019 Accuracy: 0.9631 Time: 10.16093  | Val Loss: 0.2361 Accuracy: 0.9438\n",
      "Epoch: 118/160 Train Loss: 0.1029 Accuracy: 0.9637 Time: 10.19305  | Val Loss: 0.2376 Accuracy: 0.9433\n",
      "Epoch: 119/160 Train Loss: 0.1017 Accuracy: 0.9655 Time: 10.20032  | Val Loss: 0.2372 Accuracy: 0.9428\n",
      "Epoch: 120/160 Train Loss: 0.1041 Accuracy: 0.9634 Time: 10.24293  | Val Loss: 0.2372 Accuracy: 0.9423\n",
      "Epoch: 121/160 Train Loss: 0.1038 Accuracy: 0.9635 Time: 10.23019  | Val Loss: 0.2353 Accuracy: 0.9434\n",
      "Epoch: 122/160 Train Loss: 0.1057 Accuracy: 0.9631 Time: 10.23946  | Val Loss: 0.2357 Accuracy: 0.9435\n",
      "Epoch: 123/160 Train Loss: 0.1000 Accuracy: 0.9657 Time: 10.26479  | Val Loss: 0.2340 Accuracy: 0.9429\n",
      "Epoch: 124/160 Train Loss: 0.1066 Accuracy: 0.9626 Time: 10.17079  | Val Loss: 0.2358 Accuracy: 0.9431\n",
      "Epoch: 125/160 Train Loss: 0.1022 Accuracy: 0.9641 Time: 10.15527  | Val Loss: 0.2349 Accuracy: 0.9435\n",
      "Epoch: 126/160 Train Loss: 0.1051 Accuracy: 0.9639 Time: 10.20883  | Val Loss: 0.2379 Accuracy: 0.9432\n",
      "Epoch: 127/160 Train Loss: 0.1020 Accuracy: 0.9654 Time: 10.22386  | Val Loss: 0.2351 Accuracy: 0.9428\n",
      "Epoch: 128/160 Train Loss: 0.1011 Accuracy: 0.9647 Time: 10.23098  | Val Loss: 0.2364 Accuracy: 0.9430\n",
      "Epoch: 129/160 Train Loss: 0.0995 Accuracy: 0.9648 Time: 10.17987  | Val Loss: 0.2344 Accuracy: 0.9437\n",
      "Epoch: 130/160 Train Loss: 0.1010 Accuracy: 0.9646 Time: 10.14716  | Val Loss: 0.2397 Accuracy: 0.9426\n",
      "Epoch: 131/160 Train Loss: 0.1024 Accuracy: 0.9645 Time: 10.15198  | Val Loss: 0.2365 Accuracy: 0.9428\n",
      "Epoch: 132/160 Train Loss: 0.0981 Accuracy: 0.9653 Time: 10.22590  | Val Loss: 0.2360 Accuracy: 0.9433\n",
      "Epoch: 133/160 Train Loss: 0.1026 Accuracy: 0.9642 Time: 10.19724  | Val Loss: 0.2375 Accuracy: 0.9437\n",
      "Epoch: 134/160 Train Loss: 0.1083 Accuracy: 0.9618 Time: 10.12736  | Val Loss: 0.2369 Accuracy: 0.9427\n",
      "Epoch: 135/160 Train Loss: 0.1044 Accuracy: 0.9638 Time: 10.13230  | Val Loss: 0.2376 Accuracy: 0.9431\n",
      "Epoch: 136/160 Train Loss: 0.1023 Accuracy: 0.9649 Time: 10.14702  | Val Loss: 0.2362 Accuracy: 0.9430\n",
      "Epoch: 137/160 Train Loss: 0.1026 Accuracy: 0.9641 Time: 10.21494  | Val Loss: 0.2350 Accuracy: 0.9431\n",
      "Epoch: 138/160 Train Loss: 0.1034 Accuracy: 0.9644 Time: 10.15759  | Val Loss: 0.2359 Accuracy: 0.9430\n",
      "Epoch: 139/160 Train Loss: 0.1039 Accuracy: 0.9637 Time: 10.15837  | Val Loss: 0.2374 Accuracy: 0.9432\n",
      "Epoch: 140/160 Train Loss: 0.1012 Accuracy: 0.9652 Time: 10.14760  | Val Loss: 0.2375 Accuracy: 0.9428\n",
      "Epoch: 141/160 Train Loss: 0.1023 Accuracy: 0.9645 Time: 10.12766  | Val Loss: 0.2343 Accuracy: 0.9433\n",
      "Epoch: 142/160 Train Loss: 0.1032 Accuracy: 0.9640 Time: 10.16530  | Val Loss: 0.2359 Accuracy: 0.9433\n",
      "Epoch: 143/160 Train Loss: 0.1007 Accuracy: 0.9644 Time: 10.17407  | Val Loss: 0.2362 Accuracy: 0.9429\n",
      "Epoch: 144/160 Train Loss: 0.1029 Accuracy: 0.9639 Time: 10.21601  | Val Loss: 0.2366 Accuracy: 0.9433\n",
      "Epoch: 145/160 Train Loss: 0.1009 Accuracy: 0.9648 Time: 10.24266  | Val Loss: 0.2377 Accuracy: 0.9426\n",
      "Epoch: 146/160 Train Loss: 0.1032 Accuracy: 0.9634 Time: 10.17274  | Val Loss: 0.2346 Accuracy: 0.9436\n",
      "Epoch: 147/160 Train Loss: 0.1024 Accuracy: 0.9642 Time: 10.19590  | Val Loss: 0.2368 Accuracy: 0.9432\n",
      "Epoch: 148/160 Train Loss: 0.1045 Accuracy: 0.9633 Time: 10.18048  | Val Loss: 0.2372 Accuracy: 0.9436\n",
      "Epoch: 149/160 Train Loss: 0.1026 Accuracy: 0.9640 Time: 10.16687  | Val Loss: 0.2370 Accuracy: 0.9434\n",
      "Epoch: 150/160 Train Loss: 0.1007 Accuracy: 0.9652 Time: 10.25352  | Val Loss: 0.2373 Accuracy: 0.9427\n",
      "Epoch: 151/160 Train Loss: 0.1047 Accuracy: 0.9633 Time: 10.17145  | Val Loss: 0.2354 Accuracy: 0.9436\n",
      "Epoch: 152/160 Train Loss: 0.1041 Accuracy: 0.9626 Time: 10.18054  | Val Loss: 0.2348 Accuracy: 0.9443\n",
      "Epoch: 153/160 Train Loss: 0.1043 Accuracy: 0.9642 Time: 10.17409  | Val Loss: 0.2361 Accuracy: 0.9426\n",
      "Epoch: 154/160 Train Loss: 0.1049 Accuracy: 0.9631 Time: 10.13340  | Val Loss: 0.2360 Accuracy: 0.9429\n",
      "Epoch: 155/160 Train Loss: 0.1037 Accuracy: 0.9638 Time: 10.10582  | Val Loss: 0.2383 Accuracy: 0.9425\n",
      "Epoch: 156/160 Train Loss: 0.1011 Accuracy: 0.9650 Time: 10.12135  | Val Loss: 0.2357 Accuracy: 0.9435\n",
      "Epoch: 157/160 Train Loss: 0.1034 Accuracy: 0.9640 Time: 10.25295  | Val Loss: 0.2361 Accuracy: 0.9434\n",
      "Epoch: 158/160 Train Loss: 0.1029 Accuracy: 0.9638 Time: 10.19165  | Val Loss: 0.2354 Accuracy: 0.9432\n",
      "Epoch: 159/160 Train Loss: 0.1059 Accuracy: 0.9629 Time: 10.13884  | Val Loss: 0.2367 Accuracy: 0.9435\n",
      "Epoch: 160/160 Train Loss: 0.1077 Accuracy: 0.9631 Time: 10.23161  | Val Loss: 0.2355 Accuracy: 0.9427\n",
      "#Parameter: 21282122 Accuracy: 0.9427\n"
     ]
    }
   ],
   "source": [
    "from hdd.data_util.auto_augmentation import CIFAR10Policy\n",
    "\n",
    "train_dataset_transforms = transforms.Compose(\n",
    "    [\n",
    "        transforms.RandomCrop(size=32, padding=4),\n",
    "        transforms.RandomHorizontalFlip(),\n",
    "        CIFAR10Policy(),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize(mean=TRAIN_MEAN, std=TRAIN_STD),\n",
    "    ]\n",
    ")\n",
    "# 加载数据集\n",
    "train_dataset = datasets.CIFAR10(\n",
    "    root=DATA_ROOT,\n",
    "    train=True,\n",
    "    transform=train_dataset_transforms,\n",
    "    download=True,\n",
    ")\n",
    "val_dataset = datasets.CIFAR10(\n",
    "    root=DATA_ROOT,\n",
    "    train=False,\n",
    "    transform=transforms.Compose(\n",
    "        [transforms.ToTensor(), transforms.Normalize(TRAIN_MEAN, TRAIN_STD)]\n",
    "    ),\n",
    "    download=True,\n",
    ")\n",
    "BATCH_SIZE = 128\n",
    "train_dataloader = torch.utils.data.DataLoader(\n",
    "    train_dataset,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    shuffle=True,\n",
    "    num_workers=4,\n",
    ")\n",
    "val_dataloader = torch.utils.data.DataLoader(\n",
    "    val_dataset,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    shuffle=False,\n",
    "    num_workers=4,\n",
    ")\n",
    "\n",
    "net, _ = train_net(\n",
    "    resnet34_config,\n",
    "    train_dataloader,\n",
    "    val_dataloader,\n",
    "    dropout=0.0,\n",
    "    lr=0.01,\n",
    "    weight_decay=0,\n",
    ")\n",
    "\n",
    "eval_result = eval_image_classifier(net, val_dataloader.dataset, DEVICE)\n",
    "ss = [result.gt_label == result.predicted_label for result in eval_result]\n",
    "print(f\"#Parameter: {count_trainable_parameter(net)} Accuracy: {sum(ss) / len(ss)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ed15b78",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch-cu124",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
